#coding:utf-8
from opensearch_client import OpensearchClient
import time

all_data = {}
dataid_key = {}
search_client = OpensearchClient()

total = 0

for docs in search_client.large():
    for doc in docs:
        group_id = doc['_source']['group_id']
        dataid   = doc['_source']['dataid']
        reason = doc['_source']['reason']
        origin = doc['_source']['origin']
        title = doc['_source']['title']
        classes = doc['_source']['classes']
        ctime = doc['_source']['ctime']
        str_time = time.strftime("%Y-%m-%d %H:%M", time.localtime(int(ctime)))
        total += 1
        if group_id not in all_data:
            all_data[group_id] = []
        all_data[group_id].append([dataid, classes, group_id, reason, origin, title, str_time])
        dataid_key[dataid] = title

print(total, len(all_data), len(dataid_key))

oo = open("new_group_id", "a+")        

for group_id in all_data:
    for doc in all_data[group_id]:
        dataid, classes, group_id, reason, origin, title, str_time = doc 
        ori_title = dataid_key.get(origin)
        #oo.write("\t".join([dataid, str_time, classes,group_id, reason, origin, title, ori_title]) + '\n')
        oo.write("\t".join([str_time, classes,group_id, reason,title, ori_title]) + '\n')
    oo.write("\n")

